{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# torch 求导\n",
    "\n",
    "参考 [url](https://pytorch.org/tutorials/beginner/basics/autogradqs_tutorial.html)\n",
    "\n",
    "pytorch 实现模型训练需要完整地写下训练过程，包括反向传播求梯度以及应用梯度下降算法。（06见chapter_2/03_...)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 近似求导"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.999999999994678\n"
     ]
    }
   ],
   "source": [
    "def f(x):\n",
    "    return 3. * x ** 2 + 2. * x - 1\n",
    "#近视求导，x移动eps单位，也就是离自己很近的一个点的切线\n",
    "def approximate_derivative(f, x, eps=1e-4):\n",
    "    return (f(x + eps) - f(x - eps)) / (2. * eps)\n",
    "\n",
    "print(approximate_derivative(f, 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8.999999999993236, 41.999999999994486)\n"
     ]
    }
   ],
   "source": [
    "#求偏导数,其中一个数不动，对另外一个变量求导\n",
    "def g(x1, x2):\n",
    "    return (x1 + 5) * (x2 ** 2)\n",
    "\n",
    "def approximate_gradient(g, x1, x2, eps=1e-3):\n",
    "    dg_x1 = approximate_derivative(lambda x: g(x, x2), x1, eps)\n",
    "    dg_x2 = approximate_derivative(lambda x: g(x1, x), x2, eps)\n",
    "    return dg_x1, dg_x2\n",
    "\n",
    "print(approximate_gradient(g, 2., 3.))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## torch 近似求导"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.])\n",
      "Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.\n"
     ]
    }
   ],
   "source": [
    "# 声明两个tensor x1 和 x2，允许梯度计算，使用torch的自动求导上下文计算两个tensor的梯度\n",
    "# 使用 torch.autograd.grad 计算 y = g(x1, x2) 的偏导数\n",
    "\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "    \n",
    "(dy_dx1,) = torch.autograd.grad(y, x1)\n",
    "print(dy_dx1)\n",
    "\n",
    "try:\n",
    "    (dy_dx2,) = torch.autograd.grad(y, x2)\n",
    "    print(dy_dx2)\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.]) tensor([42.])\n"
     ]
    }
   ],
   "source": [
    "# 同时求导\n",
    "\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求偏导数\n",
    "dy_dx1, dy_dx2 = torch.autograd.grad(y, [x1, x2])\n",
    "\n",
    "\n",
    "print(dy_dx1, dy_dx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.]) tensor([42.])\n"
     ]
    }
   ],
   "source": [
    "# 当然我们一般直接用 backward\n",
    "\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求偏导数    \n",
    "y.backward()\n",
    "print(x1.grad, x2.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二阶导\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([6.]) tensor([6.]) tensor([14.])\n"
     ]
    }
   ],
   "source": [
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求y对x1和x2的二阶偏导数\n",
    "dy_dx1, dy_dx2 = torch.autograd.grad(y, [x1, x2], create_graph=True)\n",
    "_, dy_dx1_dx1 = torch.autograd.grad(dy_dx1, [x1, x2], allow_unused=True)\n",
    "dy_dx2_dx1, dy_dx2_dx2 = torch.autograd.grad(dy_dx2, [x1, x2], allow_unused=True)\n",
    "print(dy_dx1_dx1, dy_dx2_dx1, dy_dx2_dx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.3333, requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#模拟梯度下降算法 SGD\n",
    "learning_rate = 0.1\n",
    "x = torch.tensor(-1.0, requires_grad=True)\n",
    "for _ in range(100):\n",
    "    z = f(x)\n",
    "    z.backward()\n",
    "    x.data.sub_(learning_rate * x.grad)\n",
    "    x.grad.zero_()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.3333, requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "#GradientTape与optimizer（优化器）结合使用\n",
    "learning_rate = 0.1\n",
    "x = torch.tensor(2.0, requires_grad=True)\n",
    "optimizer = torch.optim.SGD([x], lr=learning_rate)\n",
    "for _ in range(100):\n",
    "    z = f(x)\n",
    "    z.backward()\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "print(x)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
